import os
import re
import json
import tarfile
from base_class.base_utils import spend_time
from base_class.inspect_logging import print_to_log
from config import *
import shutil
import sys
import boto3
from datetime import datetime
from data_process import upmaterial


class Process():

    def __init__(self, processdir):
        self.filenames = []
        self.dirnames = []
        self.processdir = processdir
        self.s3 = boto3.resource('s3')
        self.bucket = bucket

    @spend_time("获取文件名耗时：")
    def get_filenames(self, filepath):
        for root, dirs, files in os.walk(filepath, topdown=False):
            for file in files:
                if 'ac' in file:
                    match = re.search(r"ac.(\d{4}-\d{1,2}-\d{1,2}.tar.gz)", file)
                else:
                    match = re.search(r"hc.(\d{4}-\d{1,2}-\d{1,2}.tar.gz)", file)
                if match:
                    day = re.search(r"(\d{4}-\d{1,2}-\d{1,2})", match.group()).group()
                    day = datetime.strptime(day, '%Y-%m-%d')
                    if day.year == 2020:
                        if day.month != 1:
                            self.filenames.append(match.group())
                    else:
                        self.filenames.append(match.group())
        return self.filenames

    @spend_time("解压耗时：")
    def unzip(self, filepath, filename):
        match = re.search(r"(\d{4}-\d{1,2}-\d{1,2})", filename)
        if match and len(filename) < 24:
            dirname = match.group()
            if not os.path.exists(os.path.join(filepath, dirname)):
                os.makedirs(os.path.join(filepath, dirname))
            tf = tarfile.open(os.path.join(filepath, filename))
            tf.extractall(os.path.join(filepath, dirname))
            self.dirnames.append(dirname)

    @spend_time("处理数据耗时：")
    def data_process(self, rootpath, childdir):
        outpathls = []
        filenames = []
        filepath = os.path.join(rootpath, childdir)
        basepath = os.path.join(rootpath,self.processdir)
        exists = os.path.exists(filepath+'.json')
        if exists:
            for root, dirs, files in os.walk(basepath, topdown=False):
                for file in files:
                    outpathls.append(os.path.join(basepath, file))
                    filenames.append(file)
            return outpathls, filenames
        else:
            if not os.path.exists(basepath):
                os.makedirs(basepath)
            for root, dirs, files in os.walk(filepath, topdown=False):
                outpath = os.path.join(basepath, childdir+".json")
                outpathls.append(outpath)
                filenames.append(childdir+".json")
                outFile = open(outpath, "w", encoding='utf-8')
                for file in files:
                    readpath = os.path.join(root, file)
                    with open(readpath, mode='rb') as f:
                        line = f.readline()
                        while line:
                            if isinstance(line, bytes):
                                line = line.decode(encoding="utf8", errors="ignore")
                            try:
                                js_str = json.dumps(strprocess(line), ensure_ascii=False)
                                outFile.write(js_str + ',\n')
                            except Exception as e:
                                print(e)
                                print(line)
                            line = f.readline()
                outFile.close()
            shutil.rmtree(filepath)
            return outpathls, filenames


    @spend_time("上传文件耗时：")
    def upload(self, bucket=None, filename=None, pathfilename=None):
        try:
            if bucket and filename and pathfilename:
                self.s3.Object(bucket, filename).upload_file(pathfilename)
                os.remove(pathfilename)
            print_to_log(SUCCESS)
        except Exception as e:
            print_to_log(e)
            print_to_log(PROGRAMERROR)


def strprocess(str):
    result = {}
    match = re.findall("({.*})", str)
    try:
        clean = match[0].replace("{","").replace("}","")
        ls = clean.split(",")
        for vchar in ls:
            minls = vchar.split("=")
            result[minls[0].strip()] =minls[1].replace("null","").replace("'","").replace("--","").strip()
        for filterkey in filterkeys:
            result.pop(filterkey, None)
        return result
    except Exception as e:
        print_to_log(e)
        return None


if __name__ == '__main__':
    s3 = boto3.resource('s3')
    filePath = None
    try:
        filePath = sys.argv[1]
    except:
        filePath = r"/backup3/backup34/DeviceData/Fridge/2022/"
    process = Process("processdir")
    filenames = process.get_filenames(filePath)
    results = []
    for filename in filenames:
        upmaterial(filePath, filename, process, filePath)












